package kevin.bigdata.spark.wc

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

/**
 * @author: kevin  
 * @date: 11:16 2021/3/26
 */
object Spark01_WordCount {
  def main(args: Array[String]): Unit = {
    // application
    // spark framework

    // TODO 建立和SPark框架的连接
    // JDBC：Connection
    val sparConf = new SparkConf().setMaster("local").setAppName("WordCount").set("spark.testing.memory","2147480000")
    val sc = new SparkContext(sparConf)

    // TODO 执行业务操作

    // 1.读取文件，获取行数据
    val lines: RDD[String] = sc.textFile("data")
    // 2.将行数据进行拆分，拆成单个单词 "hello world" =>hello, world, hello, world
    val words: RDD[String] = lines.flatMap(_.split(" "))
    // 3.将数据根据单词进行分组，便于统计 (hello,hello),(word,word)
    val wordGroup: RDD[(String, Iterable[String])] = words.groupBy(word=>word)
    // 4.对分组后的数据进行转换(hello, 2),(word, 2)
    val wordToCount = wordGroup.map{
      case (word, list) => {
        (word, list.size)
      }
    }
    // 5. 将转换结果采集到控制台打印出来
    val array: Array[(String, Int)] = wordToCount.collect()
    array.foreach(println)
    // TODO 关闭连接
    sc.stop()
  }
}
